{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 모듈 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.sparse import csr_matrix\n",
    "from scipy.sparse import issparse\n",
    "from sklearn import datasets\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "import random\n",
    "import math\n",
    "from numpy import linalg as la"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 데이터 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "files = ['data/scene_train', 'data/scene_test']\n",
    "\n",
    "data = datasets.load_svmlight_files(files, multilabel=True)\n",
    "train_data = data[0]\n",
    "train_target = np.array(MultiLabelBinarizer().fit_transform(data[1]))\n",
    "test_data = data[2]\n",
    "test_target = data[3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. 클래스 생성하기 - 학습, 테스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class BPMLL:\n",
    "    \n",
    "    def __init__(self, neural=0.2, epoch=20, weight_decay=0.00001, regularization=0.1, print_procedure=False):\n",
    "        self.features = 0\n",
    "        self.classes = 0\n",
    "        self.samples = 0\n",
    "        self.neural_num = 0\n",
    "        self.learn_rate = 0.05\n",
    "        self.neural_percent = neural\n",
    "        self.epoch = epoch\n",
    "        self.weightsDecayCost = weight_decay\n",
    "        self.regularization = regularization\n",
    "        self.error_small_change = 0.00001\n",
    "        self.final_error = 0\n",
    "        self.dataset = []\n",
    "        self.threshold = None\n",
    "        self.wsj_matrix = []\n",
    "        self.vhs_matrix = []\n",
    "        self.bias_b = []\n",
    "        self.bias_a = []\n",
    "        self.print_procedure = print_procedure\n",
    "        self.trained = False\n",
    "\n",
    "    def fit(self, x, y):\n",
    "        self.features = x.shape[1]  \n",
    "        self.classes = y.shape[1]  \n",
    "        self.dataset = self.prepare_data(x, y)\n",
    "        self.samples = len(self.dataset)\n",
    "        self.neural_num = int(self.features * self.neural_percent)   \n",
    "        self.wsj_matrix = np.random.random_sample((self.neural_num, self.classes)) - 0.5  \n",
    "        self.vhs_matrix = np.random.random_sample((self.features, self.neural_num)) - 0.5  \n",
    "        self.bias_b = np.ones((1, self.classes))  \n",
    "        self.bias_a = np.ones((1, self.neural_num))  \n",
    "        self.iterate_training()\n",
    "        self.trained = True\n",
    "        return self\n",
    "\n",
    "    def prepare_data(self, x, y):\n",
    "        dataset = [] \n",
    "        for i in range(x.shape[0]):\n",
    "            dataset.append(TrainPair(x[i], y[i]))\n",
    "        return dataset\n",
    "\n",
    "    def iterate_training(self):\n",
    "        prev_error = self.global_error()\n",
    "        for ep in range(self.epoch):\n",
    "            if self.print_procedure:\n",
    "                print(\"학습된 데이터의 epoch는 \" + str(ep))\n",
    "            for i in range(self.samples):\n",
    "                self.fit_once(i)\n",
    "            error = self.global_error()\n",
    "            diff = prev_error - error\n",
    "            if diff <= self.error_small_change * prev_error:\n",
    "                self.build_threshhold()\n",
    "                self.final_error = error\n",
    "                return\n",
    "            prev_error = error\n",
    "\n",
    "        self.build_threshhold()\n",
    "        self.final_error = prev_error\n",
    "        return\n",
    "\n",
    "    def fit_once(self, index):\n",
    "        x = self.dataset[index].attributes\n",
    "        y = self.dataset[index].labels\n",
    "        x_vec = np.array([x]).T\n",
    "        is_label = self.dataset[index].isLabel\n",
    "        not_label = self.dataset[index].notLabel\n",
    "        is_label_length = len(is_label)\n",
    "        not_label_length = len(not_label)\n",
    "        b, c = self.forward_propagation(x)\n",
    "        exp_func = math.exp\n",
    "        dj_sigma = np.zeros((1, self.classes))\n",
    "        for j in range(self.classes):\n",
    "            tmp = 0\n",
    "            if y[j] == 1:\n",
    "                for l in not_label:\n",
    "                    tmp += exp_func(-(c[0, j] - c[0, l]))\n",
    "            else:\n",
    "                for k in is_label:\n",
    "                    tmp -= exp_func(-(c[0, k] - c[0, j]))\n",
    "            dj_sigma[0, j] = tmp\n",
    "        \n",
    "        d = (1 / (is_label_length * not_label_length)) * dj_sigma * (1 - np.square(c))\n",
    "        b_vec = b.T\n",
    "        d_vec = d.T\n",
    "        es_sigma_vec = np.dot(self.wsj_matrix, d_vec)\n",
    "        e_vec = es_sigma_vec * (1 - np.square(b_vec))\n",
    "        e = e_vec.T\n",
    "        self.wsj_matrix = (1 - self.weightsDecayCost) * self.wsj_matrix + self.learn_rate * np.dot(b_vec, d)\n",
    "        self.vhs_matrix = (1 - self.weightsDecayCost) * self.vhs_matrix + self.learn_rate * np.dot(x_vec, e)\n",
    "        self.bias_b = (1 - self.weightsDecayCost) * self.bias_b + self.learn_rate * d\n",
    "        self.bias_a = (1 - self.weightsDecayCost) * self.bias_a + self.learn_rate * e\n",
    "\n",
    "        return\n",
    "\n",
    "    def forward_propagation(self, x):\n",
    "        x = np.array([x])\n",
    "        netb = np.dot(x, self.vhs_matrix) + self.bias_a\n",
    "        b = tanh(netb)   \n",
    "        netc = np.dot(b, self.wsj_matrix) + self.bias_b\n",
    "        c = tanh(netc)\n",
    "        return b, c\n",
    "\n",
    "    def global_error(self):\n",
    "        global_error = 0\n",
    "        weights_square_sum = np.sum(np.square(self.wsj_matrix)) + np.sum(\n",
    "                np.square(self.vhs_matrix)) + np.sum(np.square(self.bias_b)) + np.sum(np.square(self.bias_a))\n",
    "        for i in range(self.samples):\n",
    "            c = self.forward_propagation(self.dataset[i].attributes)[1]\n",
    "            yi = self.dataset[i].isLabel\n",
    "            nyi = self.dataset[i].notLabel\n",
    "            yi_length = len(yi)\n",
    "            nyi_length = len(nyi)\n",
    "            A = np.array([[c[0, l] - c[0, k] for k in yi] for l in nyi])\n",
    "            global_error += 1 / (yi_length * nyi_length) * np.sum(np.exp(A))\n",
    "        global_error += self.regularization * weights_square_sum\n",
    "        return global_error\n",
    "\n",
    "    def build_threshhold(self):\n",
    "        model_outputs = []\n",
    "        ideal_labels = []\n",
    "        for i in range(self.samples):\n",
    "            c = self.forward_propagation(self.dataset[i].attributes)[1][0] \n",
    "            print(c)\n",
    "            model_outputs.append(c)\n",
    "            ideal_labels.append(self.dataset[i].labels)\n",
    "        self.threshold = ThresholdFunction(model_outputs, ideal_labels)\n",
    "\n",
    "    def predict(self, x, rank_results=False):\n",
    "        \n",
    "        samples, features = x.shape\n",
    "        result = RankResults()\n",
    "        for sample_index in range(samples):\n",
    "            sample_result = []\n",
    "            c = self.forward_propagation(x[sample_index])[1][0]\n",
    "            print(\"ghihi\")\n",
    "            print(c)\n",
    "            threshold = self.threshold.compute_threshold(c)\n",
    "            print(threshold)\n",
    "            top_label = None\n",
    "            max_value = 0\n",
    "            count = 0\n",
    "            for j in range(self.classes):\n",
    "                if c[j] >= threshold:\n",
    "                    count += 1\n",
    "                    sample_result.append(j)\n",
    "                if c[j] > max_value:\n",
    "                    top_label = j\n",
    "                    max_value = c[j]\n",
    "            if count == 0:\n",
    "                sample_result.append(top_label)\n",
    "            result.add(sample_result, top_label, c)\n",
    "        if rank_results is False:\n",
    "            result = result.predictedLabels\n",
    "\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TrainPair:\n",
    "    def __init__(self, attributes, labels):\n",
    "        self.attributes = attributes \n",
    "        self.labels = labels \n",
    "        self.isLabel = [] \n",
    "        self.notLabel = [] \n",
    "        for j in range(labels.shape[0]):\n",
    "            if labels[j] == 1:\n",
    "                self.isLabel.append(j)\n",
    "            else:\n",
    "                self.notLabel.append(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ThresholdFunction:\n",
    "\n",
    "    def __init__(self, model_output, ideal_labels):\n",
    "        self.parameters = []\n",
    "        self.build(model_output, ideal_labels)\n",
    "\n",
    "    def build(self, model_output, ideal_labels):\n",
    "        samples = len(ideal_labels)\n",
    "        labels = len(ideal_labels[0]) \n",
    "        threshholds = np.zeros(samples) \n",
    "\n",
    "        for sample_index in range(samples): \n",
    "            label_value = [float('inf') for i in range(labels)]\n",
    "            notlabel_value = [float('-inf') for i in range(labels)] \n",
    "            for j in range(labels):\n",
    "                if ideal_labels[sample_index][j] == 1:  \n",
    "                    label_value[j] = model_output[sample_index][j]   \n",
    "                else:\n",
    "                    notlabel_value[j] = model_output[sample_index][j]\n",
    "       \n",
    "            label_min = min(label_value)\n",
    "            notlabel_max = max(notlabel_value)\n",
    "\n",
    "            if label_min != notlabel_max:\n",
    "                if label_min == float('inf'):\n",
    "                    threshholds[sample_index] = notlabel_max + 0.1\n",
    "                elif notlabel_max == float('-inf'):\n",
    "                    threshholds[sample_index] = label_min - 0.1\n",
    "                else:\n",
    "                    threshholds[sample_index] = (label_min + notlabel_max) / 2\n",
    "            else:\n",
    "                threshholds[sample_index] = label_min\n",
    "        \n",
    "        model_output = np.concatenate((model_output, np.array([np.ones(samples)]).T), axis=1)\n",
    "        self.parameters = np.linalg.lstsq(model_output, threshholds)[0]\n",
    "\n",
    "    def compute_threshold(self, outputs):\n",
    "        parameter_length = len(self.parameters)\n",
    "        b_index = parameter_length - 1\n",
    "\n",
    "        threshold = 0\n",
    "        for i in range(b_index):\n",
    "            threshold += outputs[i] * self.parameters[i]\n",
    "        threshold += self.parameters[b_index]\n",
    "        #print(threshold)\n",
    "        return threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tanh(x):\n",
    "    return 2 / (1 + np.exp(-2 * x)) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class RankResults:\n",
    "    def __init__(self):\n",
    "        self.predictedLabels = []\n",
    "        self.topRankedLabels = []\n",
    "        self.outputs = []\n",
    "\n",
    "    def add(self, predict_set, top_label, output):\n",
    "        self.predictedLabels.append(predict_set)\n",
    "        self.topRankedLabels.append(top_label)\n",
    "        self.outputs.append(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. 벡터화 및 실시"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "shapes (1,) and (294,117) not aligned: 1 (dim 0) != 294 (dim 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-0fe4aee8802b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBPMLL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprint_procedure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mneural\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m40000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrank_results\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-17-289ba8fa5320>\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y)\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias_b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias_a\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneural_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterate_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrained\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-17-289ba8fa5320>\u001b[0m in \u001b[0;36miterate_training\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0miterate_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m         \u001b[0mprev_error\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mep\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_procedure\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-17-289ba8fa5320>\u001b[0m in \u001b[0;36mglobal_error\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    108\u001b[0m                 np.square(self.vhs_matrix)) + np.sum(np.square(self.bias_b)) + np.sum(np.square(self.bias_a))\n\u001b[1;32m    109\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m             \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_propagation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattributes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    111\u001b[0m             \u001b[0myi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misLabel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m             \u001b[0mnyi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnotLabel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-17-289ba8fa5320>\u001b[0m in \u001b[0;36mforward_propagation\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     97\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward_propagation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0mnetb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvhs_matrix\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias_a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m         \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnetb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0mnetc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwsj_matrix\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias_b\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: shapes (1,) and (294,117) not aligned: 1 (dim 0) != 294 (dim 0)"
     ]
    }
   ],
   "source": [
    "result = BPMLL(print_procedure=True, neural=0.4, regularization=0, epoch=40000).fit(train_data, train_target).predict(train_data, rank_results=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Aggregate:\n",
    "    @staticmethod\n",
    "    def intersection(a, b):\n",
    "        inter = 0\n",
    "        for i in a:\n",
    "            if i in b:\n",
    "                inter += 1\n",
    "        return inter\n",
    "\n",
    "    @staticmethod\n",
    "    def sum(a, b):\n",
    "        return len(a) + len(b) - Aggregate.intersection(a, b)\n",
    "\n",
    "    @staticmethod\n",
    "    def sym_difference(a, b):\n",
    "        return len(a) + len(b) - 2 * Aggregate.intersection(a, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class UniversalMetrics:\n",
    "    def __init__(self, expected, predicted):\n",
    "        self.sampleNum = len(expected)\n",
    "        self.expectedLabels = [[int(i) for i in expected[j]] for j in range(len(expected))]\n",
    "        # fix for divide by zero problems, this will not affect the final result\n",
    "        for predict_index in range(len(predicted)):\n",
    "            if len(predicted[predict_index]) == 0:\n",
    "                predicted[predict_index].append(None)\n",
    "        self.predictedLabels = predicted\n",
    "\n",
    "    def accuracy(self):\n",
    "        result = 0\n",
    "        for index in range(self.sampleNum):\n",
    "            expected = self.expectedLabels[index]\n",
    "            predicted = self.predictedLabels[index]\n",
    "            result += Aggregate.intersection(expected, predicted) / Aggregate.sum(expected, predicted)\n",
    "        return result / self.sampleNum\n",
    "\n",
    "    def precision(self):\n",
    "        result = 0\n",
    "        for index in range(self.sampleNum):\n",
    "            expected = self.expectedLabels[index]\n",
    "            predicted = self.predictedLabels[index]\n",
    "            result += Aggregate.intersection(expected, predicted) / len(predicted)\n",
    "        return result / self.sampleNum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class RankResults:\n",
    "    def __init__(self):\n",
    "        self.predictedLabels = []\n",
    "        self.topRankedLabels = []\n",
    "        self.outputs = []\n",
    "\n",
    "    def add(self, predict_set, top_label, output):\n",
    "        self.predictedLabels.append(predict_set)\n",
    "        self.topRankedLabels.append(top_label)\n",
    "        self.outputs.append(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class RankMetrics(UniversalMetrics):\n",
    "    \"\"\" Metrics design for ranking systems\"\"\"\n",
    "\n",
    "    def __init__(self, expected, result):\n",
    "        self.sampleNum = len(expected)\n",
    "        expectedLabels = [[int(i) for i in expected[j]] for j in range(len(expected))]\n",
    "\n",
    "        super().__init__(expectedLabels, result.predictedLabels)\n",
    "\n",
    "        self.topRankedLabels = result.topRankedLabels\n",
    "        self.outputs = result.outputs\n",
    "        self.possibleLabelNum = len(self.outputs[0])\n",
    "\n",
    "        self.ap_prepared = False\n",
    "        self.ap = None\n",
    "        self.rl_prepared = False\n",
    "        self.rl = None\n",
    "\n",
    "    def hamming_loss(self):\n",
    "        diff_sum = 0\n",
    "        for i in range(self.sampleNum):\n",
    "            labels_sum = len(self.expectedLabels[i])\n",
    "            intersection = 0\n",
    "            for label in self.predictedLabels[i]:\n",
    "                if label in self.expectedLabels[i]:\n",
    "                    intersection += 1\n",
    "            diff_sum += labels_sum - intersection\n",
    "\n",
    "        return diff_sum / (self.possibleLabelNum * self.sampleNum)\n",
    "\n",
    "    def one_error(self):\n",
    "        error_sum = 0\n",
    "        for i in range(self.sampleNum):\n",
    "            if self.topRankedLabels[i] not in self.expectedLabels[i]:\n",
    "                error_sum += 1\n",
    "\n",
    "        return error_sum / self.sampleNum\n",
    "\n",
    "    def coverage(self):\n",
    "        cover_sum = 0\n",
    "        for i in range(self.sampleNum):\n",
    "            label_outputs = []\n",
    "            for label in self.expectedLabels[i]:\n",
    "                label_outputs.append(self.outputs[i][label])\n",
    "            min_output = min(label_outputs)\n",
    "            for j in range(self.possibleLabelNum):\n",
    "                if self.outputs[i][j] >= min_output:\n",
    "                    cover_sum += 1\n",
    "\n",
    "        return (cover_sum / self.sampleNum) - 1\n",
    "\n",
    "    def ranking_loss(self):\n",
    "        if self.rl_prepared is True:\n",
    "            return self.rl\n",
    "\n",
    "        rloss_sum = 0\n",
    "        ap_sum = 0\n",
    "        for sample_index in range(self.sampleNum):\n",
    "            unodered_part = []\n",
    "            expected_num = len(self.expectedLabels[sample_index])\n",
    "\n",
    "            sample_output = self.outputs[sample_index]\n",
    "            output_dic = {}\n",
    "            for output_index in range(self.possibleLabelNum):\n",
    "                output_dic[output_index] = sample_output[output_index]\n",
    "\n",
    "            sorted_output = sorted(output_dic.items(), key=operator.itemgetter(1), reverse=True)\n",
    "\n",
    "            temp_count = 0\n",
    "            times = 0\n",
    "            for sorted_tuples in sorted_output:\n",
    "                if times == expected_num:\n",
    "                    break\n",
    "\n",
    "                if sorted_tuples[0] not in self.expectedLabels[sample_index]:\n",
    "                    temp_count += 1\n",
    "                else:\n",
    "                    unodered_part.append(temp_count)\n",
    "                    temp_count = 0\n",
    "                    times += 1\n",
    "            if len(unodered_part) != expected_num:\n",
    "                raise Exception(\"function error for RankingLoss\")\n",
    "\n",
    "            pairs_num = 0\n",
    "            fraction_sum = 0\n",
    "            fraction_divide = 0\n",
    "            for cal_index in range(expected_num):\n",
    "                pairs_num += unodered_part[cal_index] * (expected_num - cal_index)\n",
    "                # prepare for calculating average precision\n",
    "                fraction_divide += unodered_part[cal_index] + 1\n",
    "                fraction_sum += (cal_index + 1) / fraction_divide\n",
    "\n",
    "            rloss_sum += pairs_num / (expected_num * (self.possibleLabelNum - expected_num))\n",
    "            ap_sum += fraction_sum / expected_num\n",
    "\n",
    "        self.ap = ap_sum / self.sampleNum\n",
    "        self.rl = rloss_sum / self.sampleNum\n",
    "        self.ap_prepared = True\n",
    "        self.rl_prepared = True\n",
    "\n",
    "        return self.rl\n",
    "\n",
    "    def average_precision(self):\n",
    "        # contained in the ranking_loss function to save running time\n",
    "        if self.ap_prepared is True:\n",
    "            return self.ap\n",
    "        else:\n",
    "            self.ranking_loss()\n",
    "            return self.ap\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test_target' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-2bdfdbba9068>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mwhen\u001b[0m \u001b[0myou\u001b[0m \u001b[0mspecify\u001b[0m \u001b[0mrank_results\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpredict\u001b[0m \u001b[0mfunctions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \"\"\"\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mmetric\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRankMetrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_target\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'hamming loss:'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetric\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhamming_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'one error:'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetric\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mone_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'test_target' is not defined"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "    special metric for rank systems like BPMLL and RankingSVM, you can use RankMetrics\n",
    "    when you specify rank_results=True in predict functions\n",
    "\"\"\"\n",
    "metric = RankMetrics(test_target, result)\n",
    "print('hamming loss:' + str(metric.hamming_loss()))\n",
    "print('one error:' + str(metric.one_error()))\n",
    "print('coverage:' + str(metric.coverage()))\n",
    "print('ranking_loss:' + str(metric.ranking_loss()))\n",
    "print('average_precision:' + str(metric.average_precision()))\n",
    "print('precision:' + str(metric.precision()))\n",
    "print('accuracy:' + str(metric.accuracy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-2ff358dc3598>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# feature extraction using PCA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mfeature_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mpca\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPCA\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_components\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature_size\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m//\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtrain_data_trans\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcsr_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpca\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtest_data_trans\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcsr_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpca\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'train_data' is not defined"
     ]
    }
   ],
   "source": [
    "# feature extraction using PCA\n",
    "feature_size = train_data.shape[1]\n",
    "pca = PCA(n_components=(feature_size * 10) // 100)\n",
    "train_data_trans = csr_matrix(pca.fit_transform(train_data.todense()))\n",
    "test_data_trans = csr_matrix(pca.transform(test_data.todense()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:ml_python]",
   "language": "python",
   "name": "conda-env-ml_python-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
